import json
import os
import re
import sys
import time
from ctypes import windll, c_char_p
from urllib import parse

import facade
import requests
from xjlibrary.mdatetime.mtime2 import MDateTimeUtils
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseUrl import BaseUrl

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
dirPath = BaseDir.get_new_path(TopPath, "download", "EI", "download", "json", "refs")
BaseDir.create_dir(dirPath)
dbpath = BaseDir.get_upper_dir(curPath, -1)

"""
单线程版
"""
class DiscernCode(object):
    """
    将识别验证码封装成一个类
    """

    def __init__(self, logger):
        self.appId = 6249  # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
        self.appKey = b'439aa4305465df04545dad067bb31751'  # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
        self.YDMApi = windll.LoadLibrary('yundamaAPI-x64')
        # 注意这里是普通会员账号，不是开发者账号，注册地址 http://www.yundama.com/index/reg/user
        # 开发者可以联系客服领取免费调试题分
        # self.username = b'xujiang'
        # self.password = b'xujiang1994323'
        self.username = b'office'
        self.password = b'officeHelper$123'
        # 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
        self.codetype = 3006
        # 分配30个字节存放识别结果
        self.result = c_char_p(b"                              ")
        # 验证码文件路径
        # self.filename = jpgpath.encode('utf-8')
        self.logger = logger
        # 识别超时时间 单位：秒
        self.timeout = 60

    def easy_decode_by_path(self, jpgpath):
        self.logger.info("正在开始一键识别{}".format(jpgpath))
        # 一键识别函数，无需调用 YDM_SetAppInfo 和 YDM_Login，适合脚本调用
        captchaId = self.YDMApi.YDM_EasyDecodeByPath(self.username,
                                                     self.password,
                                                     self.appId,
                                                     self.appKey,
                                                     jpgpath,
                                                     self.codetype,
                                                     self.timeout,
                                                     self.result)

        self.logger.info("一键识别：验证码ID：%d，识别结果：%s" % (captchaId, self.result.value))
        if int(captchaId) > 0:
            return True, str(self.result.value)
        else:
            self.logger.error("识别失败 请检查原因")
            return False, ""


class DownRefs(object):
    # 可用ip
    listip = [
        "192.168.30.176:8207",
        "192.168.30.176:8012",
        "192.168.30.176:8021",
        "192.168.30.36:8041",  ##
        "192.168.30.176:8076",
        "192.168.30.36:8082",  ##
        "192.168.30.176:8165",
        "192.168.30.176:8171",
        "192.168.30.36:8182",  ##
        "192.168.30.176:8195",
        "192.168.30.176:8243",
        "192.168.30.176:8031",

    ]

    def __init__(self):
        self.configfile = BaseDir.get_new_path(dbpath, "db.ini")
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(self.configfile, "db", logger=self.logger)
        self.Proxies = None
        self.discern_code_time = int(time.time()) - 300
        self.sn = requests.Session()
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.sn.headers['User-Agent'] = self.UserAgent
        self.t = int(round(time.time() * 1000))
        self.SEARCHID = ""
        self.init_form()

    def set_proxy(self, proxy):
        self.Proxies = {
            "http": proxy,
            "https": proxy  # key是指目标网站的协议
        }

    def init_form(self):
        self.Form = {
            "usageOrigin": "searchform",
            "usageZone": "expertsearch",
            "editSearch": "",
            "isFullJsonResult": "true",
            "angularReq": "true",
            "CID": "searchSubmit",
            "searchtype": "Expert",
            "origin": "searchform",
            "category": "expertsearch",
            "searchWord1": "((1994*) WN AN)",
            "allDb": "1",
            "database": "1",
            "yearselect": "yearrange",
            "startYear": "1884",
            "endYear": "2020",
            "updatesNo": "1",  # "4",
            "sort": "yr",  # "sort": "relevance",  yr代表最新 relevance相关度排序
            "autostem": "true",
            "searchStartTimestamp": str(int(round(time.time() * 1000))),
            "_": str(self.t)
        }
        # ref 请求 为get请求   self.refsgetdata 为请求参数
        self.refsurl = "https://www.engineeringvillage.com/search/doc/refs.url"
        self.refsgetdata = {
            "content": "true",
            "refType": "compendex",
            "searchtype": "Expert",
            "usageOrigin": "recordpage",
            "usageZone": "abstracttab",
            "pageType": "expertSearch",
            "SEARCHID": "07c12a411884442687a6e19f410cdf13",
            "DOCINDEX": "1",
            "database": "1",
            "docid": "cpx_M2a23a19e16b1f38b06bM72b710178163167",
            "totalResultsCount": "1",
            "displayPagination": "yes",
            "dbid": "cpx"
        }
        # 获取剩下所有的引文
        self.refsshowall = {
            "content": "true",
            "compendexajax": "t",
            "docid": "cpx_M2a23a19e16b1f38b06bM72b710178163167",
            "SEARCHID": "07c12a411884442687a6e19f410cdf13",
            "database": "1",
            "DOCINDEX": "",
            "currPageNumber": "2",
            "searchtype": "Expert",
            "pageSize": "25"
        }

        self.headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Host": "www.engineeringvillage.com",
            "Referer": "https://www.engineeringvillage.com/search/expert.url",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest"
        }

    def discern_code(self, jpgpath):
        """
        识别验证码通过云打码
        :return:
        """
        if int(time.time()) - self.discern_code_time > 60 * 5:  #
            boolresult, code = DiscernCode(self.logger).easy_decode_by_path(jpgpath)
            BaseDir.single_add_file("./code.txt", "***code is :{}\n".format(code))
            self.discern_code_time = int(time.time())
            if boolresult:
                code = code.replace("b'", "").replace("'", "")
                return True, code
            else:
                return False, ""
        else:
            self.logger.info("时间还没到休息60秒 差时为{} 应该相隔5分钟".format(int(time.time()) - self.discern_code_time))
            time.sleep(60)
            return False, ""

    def get_searchid(self):
        self.logger.info('DownOneref')

        # 如果目录存在表示下载过 但也有可能是下载一部分后中断了
        # outDir = os.path.join(dirPath, ac)
        # if os.path.exists(outDir):
        #     return

        def init_home():
            url = r'https://www.engineeringvillage.com/home.url'
            header = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                "Host": "www.engineeringvillage.com",
                "Upgrade-Insecure-Requests": "1",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
            }
            BoolResult, errString, r = facade.BaseRequest(url,
                                                          sn=self.sn,
                                                          headers=header,
                                                          verify=False,
                                                          proxies=self.Proxies,
                                                          timeout=60)
            self.logger.info(errString)
            if not BoolResult:
                return False
            self.logger.info("首页访问完成")
            print(r.url)
            BaseDir.single_write_file(r.text, "./test.html")
            if r.text.find("Chongqing University Library, General Access") > -1:
                obj = re.search(r'csrfToken\.value = "(.*?)"', r.text)
                csrfToken = obj.group(1)
                print(csrfToken)
                url = "https://www.engineeringvillage.com/customer/authenticate.url"
                postdate = {
                    "path_choice": "13022631",
                    "remember_path_flag": "true",
                    "origin": "pathChoice",
                    "zone": "main",
                    "auth_type": "CANDIDATE_PATH",
                    "csrfToken": csrfToken
                }
                BoolResult, errString, r = facade.BaseRequestPost(url=url,
                                                                  sn=self.sn,
                                                                  data=postdate,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                if BoolResult:
                    print("suceess")
                else:
                    print("failed")
                    sys.exit(-1)

        form = self.Form
        # 5846789  17977686
        form['searchWord1'] = "china WN ALL"
        # form['startYear'] = str(year)
        #         # form['endYear'] = str(year)
        url = r'https://www.engineeringvillage.com/search/submit.url'
        url = url + "?" + BaseUrl.dicts_to_url(form)
        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      headers=self.headers,
                                                      proxies=self.Proxies,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      verify=False,
                                                      timeout=60)
        self.logger.info(errString)

        # self.logger.info(r.status_code)
        if errString == "request":
            return False
        # if r and r.status_code == 200:
        #
        if not BoolResult:
            if r.status_code == 400:
                imagetextenc = json.loads(r.text)["imagetextenc"]
                url = "https://www.engineeringvillage.com/rest/captcha/image?imagetextenc={}".format(imagetextenc)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              verify=False,
                                                              timeout=60)
                if BoolResult:
                    jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                    BaseDir.single_write_wb_file(r.content, jpgpath)
                else:
                    print("获取验证码失败")
                    return False
                # code = input("please input code:")
                jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                jpgpath = jpgpath.encode('utf-8')
                bools, code = self.discern_code(jpgpath)
                if not bools:
                    return False
                url = "https://www.engineeringvillage.com/rest/captcha/verify?imagetextenc={}&userentry={}".format(
                    imagetextenc, code)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              verify=False,
                                                              timeout=60)
                if BoolResult:
                    url = r'https://www.engineeringvillage.com/search/submit.url'
                    url = url + "?" + BaseUrl.dicts_to_url(form)
                    BoolResult, errString, r = facade.BaseRequest(url=url,
                                                                  sn=self.sn,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                    if not BoolResult:
                        print("验证后失败")
                        return False
                else:
                    print("验证失败")
                    return False
            else:
                return False
        self.logger.info("搜索成功")
        print(r.url)
        BaseDir.single_write_file(r.text, "./test2.html")
        try:
            param_dict = parse.parse_qs(parse.urlparse(r.url).query)
            self.SEARCHID = param_dict['SEARCHID'][0]
            return True
        except:
            init_home()
            return False

    def down_one(self, docid, ac, refcount):
        self.refsgetdata["SEARCHID"] = self.SEARCHID
        self.refsgetdata["docid"] = docid
        # 开始下载引文
        urlpara = BaseUrl.dicts_to_url(self.refsgetdata)
        refsurl_full = self.refsurl + "?" + urlpara
        allcount = 0
        refdicts = {}
        BoolResult, errString, r = facade.BaseRequest(refsurl_full,
                                                      sn=self.sn,
                                                      mark="patrefcount",
                                                      proxies=self.Proxies,
                                                      headers=self.headers,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      verify=False,
                                                      timeout=60)

        if BoolResult:
            dicts = json.loads(r.text)
            allcount = int(dicts["result"]["abstractrecord"]["patrefcount"])
            refdicts["downdate"] = MDateTimeUtils.get_today_date_strings()
            refdicts["rawid"] = ac
            refdicts["onepage"] = r.text
        else:
            sql = "update articlenew set ref_stat=-1  where AccessionNumber='{}'".format(ac)
            self.mysqlutils.ExeSqlToDB(sql)

        if str(refcount) != str(allcount):
            sql = "update articlenew set ref_cnt='{}' where AccessionNumber='{}'".format(str(allcount), ac)
            self.mysqlutils.ExeSqlToDB(sql)

        if allcount > 25:
            self.refsshowall["SEARCHID"] = self.SEARCHID
            self.refsshowall["docid"] = docid
            urlpara = BaseUrl.dicts_to_url(self.refsshowall)
            refsurlall_full = self.refsurl + "?" + urlpara
            BoolResult, errString, r = facade.BaseRequest(refsurlall_full,
                                                          sn=self.sn,
                                                          mark="patrefcount",
                                                          proxies=self.Proxies,
                                                          headers=self.headers,
                                                          allow_redirects=True,
                                                          endstring="",
                                                          verify=False,
                                                          timeout=(30, 60))
            if BoolResult:
                dicts = json.loads(r.text)
                allcount = int(dicts["result"]["abstractrecord"]["patrefcount"])
                refdicts["twopage"] = r.text
                jsonstring = json.dumps(refdicts, ensure_ascii=False)
                BaseDir.single_add_file(BaseDir.get_new_path(dirPath, "refs.big_json"), jsonstring + "\n")
                sql = "update articlenew set ref_stat=1  where AccessionNumber='{}'".format(ac)
                self.mysqlutils.ExeSqlToDB(sql)
            else:
                sql = "update articlenew set ref_stat=-1  where AccessionNumber='{}'".format(ac)
                self.mysqlutils.ExeSqlToDB(sql)

        else:
            if str(allcount) != "-1":
                refdicts["twopage"] = ""
                jsonstring = json.dumps(refdicts, ensure_ascii=False)
                BaseDir.single_add_file(BaseDir.get_new_path(dirPath, "refs.big_json"), jsonstring + "\n")

            sql = "update articlenew set ref_stat=1  where AccessionNumber='{}'".format(ac)
            self.mysqlutils.ExeSqlToDB(sql)

    def select(self):
        sql = "SELECT AccessionNumber,docid,ref_cnt FROM `ei`.`articlenew` WHERE `country` = 'CN' AND `ref_stat` = 0 ORDER BY `ref_cnt` DESC limit 1000"
        #sql = "SELECT AccessionNumber,docid,ref_cnt FROM `ei`.`articlenew` WHERE AccessionNumber='20133816759084'"
        rows = self.mysqlutils.SelectFromDB(sql)
        return rows

    def control(self):
        self.set_proxy("192.168.30.176:8087")
        while True:
            if self.get_searchid():
                rows = self.select()
                for row in rows:
                    self.down_one(row[1], row[0], row[2])


if __name__ == "__main__":
    down = DownRefs()
    down.control()

    # down.set_proxy("192.168.30.176:8012")
    # down.down_one("cpx_6e3d601415616ba2bM426a2061377553", "20133816759084")
    # down.down_one("cpx_6e3d601415616ba2bM426a2061377553", "20133816759084")
